package practice.library.webmagic.processor.qyer

import practice.library.webmagic.persistence.SaveToCassandra
import us.codecraft.webmagic.{Page, Site}
import us.codecraft.webmagic.processor.PageProcessor
import us.codecraft.webmagic.selector.Selectable
import utility.CustomLogger

/**
  * Created by lovepocky on 16/6/20.
  */
class PrePersistenceProcessor extends PageProcessor with CustomLogger with SaveToCassandra {

  import QyerCommon._

  val site = Site.me().setRetryTimes(5).setSleepTime(1000)

  override def getSite: Site = site

  override def process(page: Page): Unit = {
    loggers.console.info(s"start to handle page: ${page.getUrl.toString}")

    val url_s = page.getUrl.toString

    if (urlFilter(url_s)) {
      loggers.console.info(s"this page will be persisted: ${url_s}")
      primarySave(url_s,page.getHtml)
    }

  }

  def urlFilter(url: String): Boolean = {
    url.matches(regexPattern.place) || url.matches(regexPattern.subPlace) || url.matches(regexPattern.allSubPlacePages("\\w+"))
  }
}
